#!/usr/bin/env perl


use strict;
use warnings;
use Data::Dumper;

my $pages = {};


sub parse {
  my ($content, $url) = @_;
  my ($title) = $content =~ /<head.*<title>(.*?)<\/title>/gis;
  my ($canonical) = $content =~ /<head.*<link\s+rel="canonical"\s+href="(.*?)".*head>/gis;
  my ($keywords) = $content =~ /<head.*<meta\s+name="keywords"\s+content="(.*?)".*head>/gis;
  my ($description) = $content =~ /<head.*<meta\s+name="description"\s+content="(.*?)".*head>/gis;
  my ($body) = $content =~ /<div\s+id="main".*<div\s+class="content">(.*?)<\/div>.*sidebar-right/gis;
  my @hrefs = $body =~ /(<a\s+href.*?\/a>)/gis;
  my @imgs = $body =~ /(<img.*?>)/gis;
  my @links = ();
  foreach my $a (@hrefs) {
    my ($link) = $a =~ /href="(.*?)"/gis;
    if ($link !~ /^\//gis and $link !~ /myfishinghome\.com/gis) {
      $link = "/" .$link;
    }
    push @links, $link;
  }

  my @images = ();
  foreach my $i (@imgs) {
    my ($image) = $i =~ /src="(.*?)"/gis;
#    if ($image !~ /^\//gis and $image !~ /myfishinghome\.com/gis) {
#     $image = "/" . $image;
#    }
    push @images, $image;
  }
  my $hash = {
    "title" => $title,
    "canonical" => $canonical,
    "keywords"  => $keywords,
    "description" => $description,
    "body" => $body,
    "hrefs" => \@links,
    "images" => \@images,
  };

  return $hash;
}

sub download
{
  my ($url) = @_;
  print "Process: $url\n";
  `wget -O file -o aa $url`;
  open F, 'file';
  local $/ = undef;
  my $content = <F>;
  binmode F;
  close F;
  return $content;
}

sub make_content
{
  my ($data) = @_;
  my $folder_name = $data->{canonical};
  $folder_name =~ s/.*?myfishinghome\.com\///gis;

  if (not -d $folder_name) {
    my $folder = '';
    my @parts = split("/", $folder_name);
    foreach my $part (@parts) {
      $folder .= $folder ? "/$part" : $part;
      mkdir $folder unless -d $folder;
      print "Create folder: " . $folder . "\n";
    }
  }


  # download images
  foreach my $i (@{$data->{images}}) {
    my $prefix = "myfishinghome.com";
    $prefix = "" if $i =~ /myfishinghome\.com/gis;
    my $download_image_name = $i;
    $download_image_name =~ s/.*?myfishinghome\.com/myfishinghome\.com/gis;
    $download_image_name = "/" . $download_image_name  unless $download_image_name =~ /^\//gis;
    my $remote_image = "$prefix$download_image_name";
    mkdir "$folder_name/images" unless -d "$folder_name/images";
    `wget -o aa -P $folder_name/images $remote_image`;
    print "Downloading: $remote_image\n";
    my $image_name = $i;
    $image_name =~ s/.*\///gis;
    $data->{body} =~ s/$i/\/$folder_name\/images\/$image_name/gis;
  }

  open F, ">$folder_name/content.html";
  print F $data->{body};
  close F;

  delete $data->{body};
  delete $data->{hrefs};
  delete $data->{images};

  $data->{type} = 'article';
  $Data::Dumper::Terse = 1;
  $Data::Dumper::Indent = 1;
  open F,  ">$folder_name/metadata.txt";
  print F Dumper($data);
  close F;
}


sub process
{
  my ($url) = @_;
  my $prefix = "myfishinghome.com";
  $prefix = "" if $url =~ /myfishinghome\.com/gis;
  $url =~ s/.*?myfishinghome\.com/myfishinghome\.com/gis;
  return if exists ($pages->{$prefix . $url});
  $pages->{$prefix . $url} = 1;
  my $page = download($prefix . $url);
  my $data = parse($page, $url);
  my $links = $data->{hrefs};
  make_content($data);
  foreach my $link (@{$links}) {
    process($link);
  }
}


MAIN:
{
  process("/all-fish-list");
}
